


from haystack.components.preprocessors import TextCleaner

text_to_clean = "1Moonlight shimmered softly, 300 Wolves howled nearby, Night enveloped everything."

cleaner = TextCleaner(convert_to_lowercase=True, remove_punctuation=False, remove_numbers=True)
result = cleaner.run(texts=[text_to_clean])
print(result)
'''
from haystack import Document
from haystack.components.preprocessors import RecursiveDocumentSplitter

chunker = RecursiveDocumentSplitter(split_length=260, split_overlap=0, separators=["\n\n", "\n", ".", " "])

doc = Document(content=text)
doc_chunks = chunker.run([doc])
print(doc_chunks["documents"])

 

from haystack import Document
from haystack.components.preprocessors import HierarchicalDocumentSplitter

doc = Document(content="This is a simple test document")
splitter = HierarchicalDocumentSplitter(block_sizes={3, 2}, split_overlap=0, split_by='word')
output=splitter.run([doc])
print(output)


from haystack import Document
from haystack.components.preprocessors import DocumentSplitter

doc = Document(content="Moonlight shimmered softly, wolves howled nearby, night enveloped everything.")

splitter = DocumentSplitter(split_by="word", split_length=2, split_overlap=0)
result = splitter.run(documents=[doc])
print(result)

from haystack import Document
from haystack.components.preprocessors import DocumentPreprocessor

doc = Document(content=f"I love pizza!" 
"" 
"helloos" 
"")
preprocessor = DocumentPreprocessor()

result = preprocessor.run(documents=[doc])
print(result["documents"])



from haystack import Document
from haystack.components.preprocessors import DocumentCleaner

doc = Document(content="This   is  a  document  to  clean\n\n\nsubstring to remove")

cleaner = DocumentCleaner(remove_substrings = ["substring to remove"])
result = cleaner.run(documents=[doc])

print(result["documents"])




from haystack import Document
from haystack.components.preprocessors import CSVDocumentSplitter

splitter = CSVDocumentSplitter(row_split_threshold=1, column_split_threshold=1)

doc = Document(
    content="""ID,LeftVal,,,RightVal,Extra
1,Hello,,,World,Joined
2,StillLeft,,,StillRight,Bridge
,,,,,
A,B,,,C,D
E,F,,,G,H
"""
)
split_result = splitter.run([doc])
print(split_result["documents"])  # List of split tables as Documents


from haystack import Document
from haystack.components.preprocessors import CSVDocumentCleaner

cleaner = CSVDocumentCleaner(ignore_rows=1, ignore_columns=0)

documents = [Document(content="""col1,col2,col3\n,,\na,b,c\n,,""" )]
cleaned_docs = cleaner.run(documents=documents)
print(cleaned_docs)

from haystack import Document
from haystack_integrations.components.preprocessors.hanlp import ChineseDocumentSplitter

# Initialize the splitter with word-based splitting
splitter = ChineseDocumentSplitter(
    split_by="word", 
    split_length=10, 
    split_overlap=3, 
    granularity="coarse"
)

# Create a Chinese document
doc = Document(content="这是第一句话，这是第二句话，这是第三句话。这是第四句话，这是第五句话，这是第六句话！")

# Warm up the component (loads the necessary models)
splitter.warm_up()

# Split the document
result = splitter.run(documents=[doc])
print(result["documents"])  # List of split documents
'''
